{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from load_corpus import load_xhj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_xhj()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "docs = [jieba.lcut(s)  for s in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(docs)]\n",
    "model = Doc2Vec(documents, vector_size=50, window=3, min_count=1, workers=8, epochs=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import heapq\n",
    "import random\n",
    "\n",
    "def process(sentence, docs):\n",
    "    sents_ = jieba.lcut(sentence)\n",
    "    vector = model.infer_vector(sents_)\n",
    "    sims = model.docvecs.most_similar([vector], topn=10)\n",
    "    return sims\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(102687, 0.8391630053520203), (148639, 0.8225857019424438), (400628, 0.8196828365325928), (570718, 0.8179580569267273), (170509, 0.8129337430000305), (50347, 0.8064911961555481), (341018, 0.805248498916626), (766266, 0.8051482439041138), (133585, 0.8029402494430542), (553056, 0.8002502918243408)]\n",
      "similarity ask: ['叫', '不', '叫', '我']\n",
      "score: 0.8391630053520203\n",
      "ans: ['我', '不想', '和', '同一个', '女人', '对', '你', '用', '同样', '的', '称呼', ' ', '你', '是', '我', '的', ' ', '是', '我', '自己', '的', '老头子']\n",
      "====\n",
      "similarity ask: ['叫', '我', '吗']\n",
      "score: 0.8225857019424438\n",
      "ans: ['为啥']\n",
      "====\n",
      "similarity ask: ['叫', '你', '小', '黄狗', '好不好']\n",
      "score: 0.8196828365325928\n",
      "ans: ['好', '~', '在', '你', '面前', '我', '就', '叫', '小', '小雪', '^', 'o', '^']\n",
      "====\n",
      "similarity ask: ['你', '为', '嘛', '叫']\n",
      "score: 0.8179580569267273\n",
      "ans: ['你', '猜', '！']\n",
      "====\n",
      "similarity ask: ['我', '不', '叫', '陈静']\n",
      "score: 0.8129337430000305\n",
      "ans: ['早就', '姓', '孙', '了', '还', '不', '承认', '！']\n",
      "====\n",
      "similarity ask: ['你', '叫', '什么', ',', '我', '什么', '时候', '叫', '你', '的']\n",
      "score: 0.8064911961555481\n",
      "ans: ['我', '是', '可爱', '的', '香香', '鸡']\n",
      "====\n",
      "similarity ask: ['叫', '你', '卖个', '萌']\n",
      "score: 0.805248498916626\n",
      "ans: ['卖萌', '什么', '的', '最', '讨厌', '了', '＞', '_', '＜']\n",
      "====\n",
      "similarity ask: ['叫', '不', '叫']\n",
      "score: 0.8051482439041138\n",
      "ans: ['必须', '的', '！']\n",
      "====\n",
      "similarity ask: ['你', '叫', '乖个', '屁']\n",
      "score: 0.8029402494430542\n",
      "ans: ['屁屁', '更', '健康', '~']\n",
      "====\n",
      "similarity ask: ['什么', '叫', '没', '了']\n",
      "score: 0.8002502918243408\n",
      "ans: ['我', '也', '想', '你', '了', '亲耐滴', '主', '~']\n",
      "====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(vec.dtype, np.int):\n"
     ]
    }
   ],
   "source": [
    "s = \"你叫什么\"\n",
    "rs = process(s, docs)\n",
    "print(rs)\n",
    "for i, score  in rs:\n",
    "    print(\"similarity ask:\",docs[i])\n",
    "    print(\"score:\", score)\n",
    "    print(\"ans:\",docs[i+1])\n",
    "    print(\"====\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from chatbot import settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "corpus_dir = settings.CORPUS_DIR\n",
    "corpus_path = os.path.join(corpus_dir, \"xiaohuangji50w_nofenci.conv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(corpus_path) as f:\n",
    "    text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = []\n",
    "for line in text.split(\"\\nE\\n\"):\n",
    "    lines.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "454131"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "p = re.compile(\"^M\\s*(?P<ask>[\\s\\S]*?)\\nM\\s*(?P<ans>[\\s\\S]*)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============\n",
      "M 好玩\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 皮鞭不好玩嘛\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 你很好玩嘛\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 你好好玩\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 就是逗你好玩\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 你挺好玩\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 好好玩\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 挺好玩的\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 挺好的\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 你是挺好玩\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 还好啦,貌似在这么回事\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 你是谁\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 组合\n",
      "M 对你很好玩！\n",
      "===============\n",
      "M 真的吗?你给我介绍个\n",
      "M 对你很好玩！\n"
     ]
    }
   ],
   "source": [
    "for line in lines:\n",
    "    if \"对你很好玩\" in line:\n",
    "        print(\"===============\")\n",
    "        print(line)\n",
    "#         m = p.search(line)\n",
    "#         if m:\n",
    "#             print(\"++++++\")\n",
    "#             print(m.group(\"ask\"))\n",
    "#             print(m.group('ans'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('不是', '那是什么？')"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.groups()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
